# -*- coding: utf-8 -*-
'''
@author: Lénia Andrade, 5607
'''
# processamento de HTML através do parser BeautifulSoup
from BeautifulSoup import BeautifulSoup
import re
import urllib2
# biblioteca para acesso através de protocolos de Internet 

class Collect_data(object):
    '''
    classdocs
    '''


#    def __init__(selfparams):
#        '''
#        Constructor
#        '''

# url of the system Rebides
    URL_REBIDES = 'http://www.rebides.oces.mctes.pt/Rebides'
    
    def connectWebpage(self, url):
        
        self.url = url 
               
        try:
            # opens the connection to the website
            file = urllib2.urlopen(self.url)
        except:
            print "Connection ERROR", self.url
            pass


        try:
            # reading data from the webpage
            data = file.read()
        except:
            print "Reading ERROR"
            pass
        else:
            file.close() 
             
            
        # processing data from the webpage
        soup = BeautifulSoup(data)
        
        return soup
    
    
    
    def get_type_of_establishment_codes(self, year):
        
        '''
        The function get the codes of higher education types of establishments  
    
        year - year which the data relate
        '''
        
        self.year = year
        
        # formation of the Universal Resource Locator
        # from the year
        
        url = self.URL_REBIDES + '0' + str(self.year)
        
        # the codes will be associated to the establishment_type 
        d_establishment_type = {}
        
        soup = self.connectWebpage(url)

        list = soup.findAll('a',)
        
        for x in list:
            # href will have the contents of the HTML reference
            href = x['href']

            # removes from href everything but the type of establishment code 
            href = re.sub('rebid\_m1\.asp\?codr\=', '', href)
        
        
            contents = x.contents
        
            # converts to utf-8
            establishment_type = contents[0].encode('utf-8')
            d_establishment_type[ establishment_type ] = href
            pass
        return d_establishment_type
    
    
    
    def get_establishment_codes(self, year, type_of_establishment_code):
        
        '''
        The function get the codes of higher education establishments
    
        year - year which the data relate
        '''
        self.year = year
        self.type_of_establishment_code = type_of_establishment_code
        
        # formation of the Universal Resource Locator
        # from the year
        url = self.URL_REBIDES + '0' + str(self.year)
        url = url + '/rebid_m1.asp?codr=' + self.type_of_establishment_code
        
        # the codes will be associated to the establishments 
        d_establishment = {}
        
        soup = self.connectWebpage(url)

        list = soup.findAll('a',)
        
        for x in list:
            #  href will have the contents of the HTML reference
            href = x['href']

            # removes from href everything but the establishment code 
            key = \
                'rebid\_m2\.asp\?CodR\={0}\&CodP\='.\
                format(type_of_establishment_code)

            href = re.sub(key, '', href)

            contents = x.contents
        
            # converts to utf-8
            establishment = contents[0].encode('utf-8')

            d_establishment[ establishment ] = href
            pass
        return d_establishment
    
    
    def get_teachers_information(self, year, type, codTypeEstablishment, establishment, codEstablishment):
        '''
        codTypeEstablishment - codigo do type de establishment
        codEstablishment - codigo do establishment
        '''
        COLLUMN_FULL_NAME = 1
        COLLUMN_GRADE = 3
        COLLUMN_COURSE = 5
        COLLUMN_CATEGORY = 7
        COLLUMN_REGIME = 9
        
        self.year = year
        self.type = type
        self.codTypeEstablishment = codTypeEstablishment
        self.establishment = establishment
        self.codEstablishment = codEstablishment

        # formacao do Universal Resource Locator
        # a partir do year
        url = self.URL_REBIDES + '0' + str(self.year)
        url = url + '/rebid_m2.asp?CodR=' + self.codTypeEstablishment + '&CodP=' + self.codEstablishment 
        
        soup = self.connectWebpage(url)
    
        list = soup.findAll('tr',)
        
        teachers_list = []

        for x in list:
            name = x.contents[COLLUMN_FULL_NAME].contents[0]
        
            grade = x.contents[COLLUMN_GRADE].contents[0]
            grade = grade.lstrip()
        
            course = x.contents[COLLUMN_COURSE].contents[0]
            course = course.lstrip()

            category = x.contents[COLLUMN_CATEGORY].contents[0]

            regime = x.contents[COLLUMN_REGIME].contents[0]

            content = x.contents[11].findAll('a')
            
            for y in content:
                # href vai ter o content da referencia em HTML
                href = y['href']

                key = 'rebid\_m3\.asp\?CodD\='
                href = re.sub(key, '', href)

                key = '\&CodP\={0}'.format(codEstablishment)
                teacher_code = re.sub(key, '', href)
                pass

            if name != "NOME COMPLETO":
                if name.encode('utf-8') != '&nbsp;':
                    full_name = name
                    full_name = full_name.expandtabs()
                    new_full = re.sub('\015', '', full_name)
                    full_name = new_full.lstrip()
                    full_name = full_name.encode('utf-8')
                
                    full_category = category.lstrip()
                    full_category = full_category.encode('utf-8')

                    full_regime = regime.lstrip()
                    full_regime = full_regime.encode('utf-8')
                
                    full_teacher_code = teacher_code
                    pass

                full_grade = grade.encode('utf-8')
                full_course = course.encode('utf-8')

                teacher_information = [ full_teacher_code,
                                      full_name,
                                      full_grade,
                                      full_course,
                                      full_category,
                                      full_regime,
                                      year,
                                      type,
                                      establishment]
                teachers_list.append(teacher_information)
                pass
            pass
        return teachers_list
